/*-
 * Copyright (c) 2006, 2007 FTS Team
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the FTS Team nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      PlaintextParser.cpp
 *
 * $FTS$
 */

// Local Includes
#include <PlaintextParser.hpp>

// C includes
#ifdef _DEBUG
  #include <stdio.h>
#endif

#include <Indexer.hpp>
#include <LemCache.hpp>
#include <DocumentStorage.hpp>
#include <DocumentBuilder.hpp>
#include <UnicodeUtils.hpp>

namespace FTSS // FullTextSearchSystem
{

CHAR_8 PlaintextParser::sWordBuffer[PlaintextParser::C_WORD_BUFFER_LEN] = {0};

//
// Constructor
//
PlaintextParser::PlaintextParser()
{
	pDocumentBuilder = new DocumentBuilder;
}

//
// Set current document zone
//
void PlaintextParser::SetZoneId(const ZoneId_t & iIZoneId) { iZoneId = iIZoneId; }

//
// Store inverted index handler and document storage
//
INT_32 PlaintextParser::SetHandlers(Indexer * pIIndexer, DocumentStorage * pIDocumentStorage, LemCache * pILemCache) throw()
{
	if (pIIndexer == NULL || pIDocumentStorage == NULL || pILemCache == NULL) { return -1; }

	pIndexer         = pIIndexer;
	pDocumentStorage = pIDocumentStorage;
	pLemCache        = pILemCache;

return 0;
}

//
// Parse part of document
//
INT_32 PlaintextParser::ParseChunk(CCHAR_P sIText, const UINT_32 & iTextLength) throw()
{
	UCHAR_P  sText    = (UCHAR_P)sIText;
	UCCHAR_P sTextEnd = (UCCHAR_P)sIText + iTextLength;
//	UCCHAR_P sWord        = NULL;
	UCHAR_P  sWordTMP     = NULL;
	UCCHAR_P sWhiteSpace  = NULL;
	UINT_32  iWordLength  = 0;
	UINT_32  iEOL = 0;

	// Skip white space
	sWhiteSpace = sText;
	while ((sText != sTextEnd) && !((*sText >= 'a' && *sText <= 'z') ||
                                        (*sText >= 'A' && *sText <= 'Z') ||
                                        (*sText >= '0' && *sText <= '9') || *sText > 127))
	{
		++sText;
	}

	if (sText == sTextEnd) { return -1; }

	// Store white space if need
	if (iFirstCallForDocument == 1)
	{
		if (sWhiteSpace != sText)
		{
			pDocumentBuilder -> AddTerm((UCCHAR_P)"", 0);
			pDocumentBuilder -> AddWhiteSpace((UCCHAR_P)sWhiteSpace, sText - sWhiteSpace);
			++iTokenPos;
		}
		iFirstCallForDocument = 0;
	}

	for(;;)
	{
		// Start of word
		//sWord = sText;
		sWordTMP = sText;
		// Word
		iWordLength = 0;
		while (sText != sTextEnd && ((*sText >= 'a' && *sText <= 'z') ||
		                             (*sText >= 'A' && *sText <= 'Z') ||
		                             (*sText >= '0' && *sText <= '9') || *sText > 127))

		{
			sText++; iWordLength++;
		}
		//fprintf(stderr, " iWordLength=%d, symbol=%c \n", iWordLength, *(sText + iWordLength - 2));
		// Add term
		pDocumentBuilder -> AddTerm(sWordTMP, iWordLength);

		// Uppercase
		sText = sWordTMP;
		UINT_64 iWordUppercaseLength = iWordLength;
		INT_32 iUnicodeError = Unicode::Uppercase((CCHAR_P)sText, iWordLength, (CHAR_P)sWordBuffer, (UINT_64)C_WORD_BUFFER_LEN, iWordUppercaseLength);
		UCHAR_P sWordUppercase = (iUnicodeError == 0 ? (UCHAR_P)sWordBuffer : (UCHAR_P)sText);
		
		sText += iWordLength;
		// Push word if end-of-stream reached
		if (sText == sTextEnd) { Token((CCHAR_P)sWordUppercase, iWordUppercaseLength); ++iTokenPos; break; }
		//fprintf(stderr, "`%.*s` ", iWordLength, sWord);
		// Push word without special symbols
		Token((CCHAR_P)sWordUppercase, iWordUppercaseLength);

		// Skip white space
		sWhiteSpace = sText; 

		++iTokenPos;
		// Skip white space
		while ((sText != sTextEnd) && !((*sText >= 'a' && *sText <= 'z') ||
						(*sText >= 'A' && *sText <= 'Z') || 
						(*sText >= '0' && *sText <= '9') || *sText > 127))
		{
			// Increase position
			if (iEOL == 0 && ('.' == *sText || '!' == *sText || '?' == *sText))
			{
				// Dot found
				pDocumentBuilder -> AddTerm((UCCHAR_P)"", 0);

				iEOL = 1;
				++iTokenPos;
			}
			sText++;
		}

		// Store white space if need
		if (sWhiteSpace != sText)
		{
			pDocumentBuilder -> AddWhiteSpace(sWhiteSpace, sText - sWhiteSpace);
		}
//fprintf(stderr, "OK, next word\n");
		if (sText == sTextEnd) { break; }
		iEOL = 0;
	}
return 0;
}

//
// Handle token;
//
void PlaintextParser::Token(CCHAR_P sToken, const UINT_32 & iTokenLength)
{
	CachedWord * aWords = pLemCache -> GetWord(sToken, iTokenLength);
//	fprintf(stderr,"sToken=%s", sToken);
//	CHAR_P bufffWord = new CHAR_8[iTokenLength];
//	for (UINT_32 iI = 0; iI <= iTokenLength; ++iI){
//		fprintf(stderr, "%c", sToken[iI]);
//		bufffWord[iI] = sToken[iI];
//	}
//	fprintf(stderr, "\n");
//	//if (bufffWord == ""i){
//		fprintf(stderr,"HUIII");
//	}

	for(INT_32 iI = 0; iI < C_MAX_FIRST_FORMS; ++iI)
	{
		if (aWords[iI].word_hash == 0) { break; }
//		if ( aWords[iI].word_hash == 8239359682090345024){
//			fprintf(stderr, "aWords[iI].word_hash=%lld ", aWords[iI].word_hash);
//			
//		}
		pIndexer -> PushWord(aWords[iI].word_hash, aWords[iI].word_form, iZoneId, iTokenPos);
	}
}

//
// Start new document
//
INT_32 PlaintextParser::StartDocument(CCHAR_P sDocName, const UINT_32 & iDocnameLength) throw()
{
	iTokenPos = 0;
	iFirstCallForDocument = 1;
	pDocumentBuilder -> StartDocument();
	return pIndexer -> StartDocument(sDocName, iDocnameLength);
}

//
// Get current token position
//
INT_32 PlaintextParser::GetTokenPos() throw() { return iTokenPos; }

//
// End current document
//
INT_32 PlaintextParser::EndDocument() throw()
{
	iTokenPos = 0;
	pIndexer -> EndDocument();

	const DocumentPart  * aParts = NULL;
	const DocumentIndex * aIndex = NULL;
	UINT_32       iPartsSize = 0;
	UINT_32       iIndexSize = 0;
	UCCHAR_P      sText      = NULL;
	UINT_32       iTextSize  = 0;

	pDocumentBuilder -> GetDocumentParts(aParts, iPartsSize, aIndex, iIndexSize, sText, iTextSize);

	DocumentId_t iDocID = pDocumentStorage -> StoreDocumentParts(aParts, iPartsSize, aIndex, iIndexSize, sText, iTextSize);

	pDocumentBuilder -> EndDocument();

return iDocID;
}

//
// A destructor
//
PlaintextParser::~PlaintextParser() throw()
{
	delete pDocumentBuilder;
}

} // namespace FTSS
// End.
